import numpy as np
import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()
import pandas as pd
from tensorflow import keras
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot
import plotly.graph_objects as go
import math
import seaborn as sns
from sklearn.metrics import mean_squared_error
np.random.seed(1)
tf.random.set_seed(1)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, GRU, Dropout, RepeatVector, TimeDistributed
from keras import backend
MODELFILENAME = 'MODELS/LSTM_6h_TFM'
TIME_STEPS=36 #6h
CMODEL = LSTM
UNITS=40
DROPOUT=0.405
ACTIVATION='tanh'
OPTIMIZER='adam'
EPOCHS=68
BATCHSIZE=45
VALIDATIONSPLIT=0.1
# Code to read csv file into Colaboratory:
# from google.colab import files
# uploaded = files.upload()
# import io
# df = pd.read_csv(io.BytesIO(uploaded['SentDATA.csv']))
# Dataset is now stored in a Pandas Dataframe
df = pd.read_csv('../../data/dadesTFM.csv')
df.reset_index(inplace=True)
df['Time'] = pd.to_datetime(df['Time'])
df = df.set_index('Time')
columns = ['PM1','PM25','PM10','PM1ATM','PM25ATM','PM10ATM']
df1 = df.copy();
df1 = df1.rename(columns={"PM 1":"PM1","PM 2.5":"PM25","PM 10":"PM10","PM 1 ATM":"PM1ATM","PM 2.5 ATM":"PM25ATM","PM 10 ATM":"PM10ATM"})
df1['PM1'] = df['PM 1'].astype(np.float32)
df1['PM25'] = df['PM 2.5'].astype(np.float32)
df1['PM10'] = df['PM 10'].astype(np.float32)
df1['PM1ATM'] = df['PM 1 ATM'].astype(np.float32)
df1['PM25ATM'] = df['PM 2.5 ATM'].astype(np.float32)
df1['PM10ATM'] = df['PM 10 ATM'].astype(np.float32)
df2 = df1.copy()
train_size = int(len(df2) * 0.8)
test_size = len(df2) - train_size
train, test = df2.iloc[0:train_size], df2.iloc[train_size:len(df2)]
train.shape, test.shape
((3117, 7), (780, 7))
#Standardize the data
for col in columns:
scaler = StandardScaler()
train[col] = scaler.fit_transform(train[[col]])
<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]])
def create_sequences(X, y, time_steps=TIME_STEPS):
Xs, ys = [], []
for i in range(len(X)-time_steps):
Xs.append(X.iloc[i:(i+time_steps)].values)
ys.append(y.iloc[i+time_steps])
return np.array(Xs), np.array(ys)
X_train, y_train = create_sequences(train[[columns[1]]], train[columns[1]])
#X_test, y_test = create_sequences(test[[columns[1]]], test[columns[1]])
print(f'X_train shape: {X_train.shape}')
print(f'y_train shape: {y_train.shape}')
X_train shape: (3081, 36, 1) y_train shape: (3081,)
#afegir nova mètrica
def rmse(y_true, y_pred):
return backend.sqrt(backend.mean(backend.square(y_pred - y_true), axis=-1))
model = Sequential()
model.add(CMODEL(units = UNITS, return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])))
model.add(Dropout(rate=DROPOUT))
model.add(TimeDistributed(Dense(1,kernel_initializer='normal',activation=ACTIVATION)))
model.compile(optimizer=OPTIMIZER, loss='mae',metrics=['mse',rmse])
model.summary()
Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= lstm (LSTM) (None, 36, 40) 6720 _________________________________________________________________ dropout (Dropout) (None, 36, 40) 0 _________________________________________________________________ time_distributed (TimeDistri (None, 36, 1) 41 ================================================================= Total params: 6,761 Trainable params: 6,761 Non-trainable params: 0 _________________________________________________________________
history = model.fit(X_train, y_train, epochs=EPOCHS, batch_size=BATCHSIZE, validation_split=VALIDATIONSPLIT,
callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, mode='min')], shuffle=False)
Epoch 1/68 62/62 [==============================] - 2s 30ms/step - loss: 0.7150 - mse: 0.8858 - rmse: 0.7219 - val_loss: 0.4889 - val_mse: 0.2797 - val_rmse: 0.5120 Epoch 2/68 62/62 [==============================] - 1s 12ms/step - loss: 0.5623 - mse: 0.6261 - rmse: 0.5975 - val_loss: 0.2919 - val_mse: 0.1325 - val_rmse: 0.3484 Epoch 3/68 62/62 [==============================] - 1s 13ms/step - loss: 0.5313 - mse: 0.5887 - rmse: 0.5749 - val_loss: 0.2437 - val_mse: 0.1025 - val_rmse: 0.3026 Epoch 4/68 62/62 [==============================] - 1s 14ms/step - loss: 0.5162 - mse: 0.5671 - rmse: 0.5646 - val_loss: 0.2225 - val_mse: 0.0898 - val_rmse: 0.2804 Epoch 5/68 62/62 [==============================] - 1s 14ms/step - loss: 0.5079 - mse: 0.5561 - rmse: 0.5584 - val_loss: 0.2095 - val_mse: 0.0824 - val_rmse: 0.2664 Epoch 6/68 62/62 [==============================] - 1s 15ms/step - loss: 0.5026 - mse: 0.5485 - rmse: 0.5543 - val_loss: 0.2048 - val_mse: 0.0787 - val_rmse: 0.2589 Epoch 7/68 62/62 [==============================] - 1s 16ms/step - loss: 0.4982 - mse: 0.5430 - rmse: 0.5504 - val_loss: 0.2008 - val_mse: 0.0757 - val_rmse: 0.2524 Epoch 8/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4953 - mse: 0.5392 - rmse: 0.5477 - val_loss: 0.1984 - val_mse: 0.0736 - val_rmse: 0.2474 Epoch 9/68 62/62 [==============================] - 1s 15ms/step - loss: 0.4925 - mse: 0.5356 - rmse: 0.5452 - val_loss: 0.1943 - val_mse: 0.0711 - val_rmse: 0.2417 Epoch 10/68 62/62 [==============================] - 1s 15ms/step - loss: 0.4907 - mse: 0.5335 - rmse: 0.5435 - val_loss: 0.1936 - val_mse: 0.0699 - val_rmse: 0.2385 Epoch 11/68 62/62 [==============================] - 1s 13ms/step - loss: 0.4894 - mse: 0.5322 - rmse: 0.5424 - val_loss: 0.1915 - val_mse: 0.0683 - val_rmse: 0.2346 Epoch 12/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4875 - mse: 0.5299 - rmse: 0.5404 - val_loss: 0.1881 - val_mse: 0.0665 - val_rmse: 0.2299 Epoch 13/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4866 - mse: 0.5290 - rmse: 0.5394 - val_loss: 0.1857 - val_mse: 0.0650 - val_rmse: 0.2262 Epoch 14/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4857 - mse: 0.5279 - rmse: 0.5386 - val_loss: 0.1856 - val_mse: 0.0644 - val_rmse: 0.2244 Epoch 15/68 62/62 [==============================] - 1s 13ms/step - loss: 0.4843 - mse: 0.5264 - rmse: 0.5373 - val_loss: 0.1816 - val_mse: 0.0626 - val_rmse: 0.2197 Epoch 16/68 62/62 [==============================] - 1s 13ms/step - loss: 0.4838 - mse: 0.5255 - rmse: 0.5364 - val_loss: 0.1815 - val_mse: 0.0621 - val_rmse: 0.2182 Epoch 17/68 62/62 [==============================] - 1s 15ms/step - loss: 0.4832 - mse: 0.5249 - rmse: 0.5360 - val_loss: 0.1810 - val_mse: 0.0616 - val_rmse: 0.2165 Epoch 18/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4824 - mse: 0.5243 - rmse: 0.5352 - val_loss: 0.1795 - val_mse: 0.0607 - val_rmse: 0.2141 Epoch 19/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4819 - mse: 0.5239 - rmse: 0.5347 - val_loss: 0.1767 - val_mse: 0.0595 - val_rmse: 0.2109 Epoch 20/68 62/62 [==============================] - 1s 16ms/step - loss: 0.4817 - mse: 0.5235 - rmse: 0.5342 - val_loss: 0.1763 - val_mse: 0.0590 - val_rmse: 0.2094 Epoch 21/68 62/62 [==============================] - 1s 15ms/step - loss: 0.4811 - mse: 0.5227 - rmse: 0.5338 - val_loss: 0.1736 - val_mse: 0.0579 - val_rmse: 0.2064 Epoch 22/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4811 - mse: 0.5229 - rmse: 0.5339 - val_loss: 0.1742 - val_mse: 0.0579 - val_rmse: 0.2060 Epoch 23/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4802 - mse: 0.5216 - rmse: 0.5329 - val_loss: 0.1718 - val_mse: 0.0570 - val_rmse: 0.2034 Epoch 24/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4803 - mse: 0.5219 - rmse: 0.5329 - val_loss: 0.1724 - val_mse: 0.0569 - val_rmse: 0.2030 Epoch 25/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4795 - mse: 0.5210 - rmse: 0.5322 - val_loss: 0.1712 - val_mse: 0.0564 - val_rmse: 0.2014 Epoch 26/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4797 - mse: 0.5212 - rmse: 0.5325 - val_loss: 0.1699 - val_mse: 0.0559 - val_rmse: 0.1998 Epoch 27/68 62/62 [==============================] - 1s 13ms/step - loss: 0.4794 - mse: 0.5207 - rmse: 0.5321 - val_loss: 0.1717 - val_mse: 0.0562 - val_rmse: 0.2004 Epoch 28/68 62/62 [==============================] - 1s 12ms/step - loss: 0.4787 - mse: 0.5202 - rmse: 0.5315 - val_loss: 0.1697 - val_mse: 0.0554 - val_rmse: 0.1982 Epoch 29/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4786 - mse: 0.5200 - rmse: 0.5315 - val_loss: 0.1685 - val_mse: 0.0550 - val_rmse: 0.1968 Epoch 30/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4785 - mse: 0.5196 - rmse: 0.5312 - val_loss: 0.1676 - val_mse: 0.0546 - val_rmse: 0.1955 Epoch 31/68 62/62 [==============================] - 1s 13ms/step - loss: 0.4782 - mse: 0.5193 - rmse: 0.5308 - val_loss: 0.1684 - val_mse: 0.0547 - val_rmse: 0.1956 Epoch 32/68 62/62 [==============================] - 1s 15ms/step - loss: 0.4781 - mse: 0.5194 - rmse: 0.5309 - val_loss: 0.1671 - val_mse: 0.0542 - val_rmse: 0.1940 Epoch 33/68 62/62 [==============================] - 1s 15ms/step - loss: 0.4779 - mse: 0.5191 - rmse: 0.5306 - val_loss: 0.1670 - val_mse: 0.0541 - val_rmse: 0.1935 Epoch 34/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4774 - mse: 0.5183 - rmse: 0.5301 - val_loss: 0.1666 - val_mse: 0.0539 - val_rmse: 0.1928 Epoch 35/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4777 - mse: 0.5188 - rmse: 0.5306 - val_loss: 0.1675 - val_mse: 0.0540 - val_rmse: 0.1929 Epoch 36/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4772 - mse: 0.5179 - rmse: 0.5298 - val_loss: 0.1663 - val_mse: 0.0535 - val_rmse: 0.1914 Epoch 37/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4772 - mse: 0.5183 - rmse: 0.5299 - val_loss: 0.1661 - val_mse: 0.0534 - val_rmse: 0.1909 Epoch 38/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4771 - mse: 0.5185 - rmse: 0.5297 - val_loss: 0.1656 - val_mse: 0.0532 - val_rmse: 0.1902 Epoch 39/68 62/62 [==============================] - 1s 13ms/step - loss: 0.4770 - mse: 0.5181 - rmse: 0.5296 - val_loss: 0.1665 - val_mse: 0.0534 - val_rmse: 0.1905 Epoch 40/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4768 - mse: 0.5179 - rmse: 0.5295 - val_loss: 0.1656 - val_mse: 0.0531 - val_rmse: 0.1895 Epoch 41/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4768 - mse: 0.5177 - rmse: 0.5295 - val_loss: 0.1651 - val_mse: 0.0529 - val_rmse: 0.1887 Epoch 42/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4765 - mse: 0.5175 - rmse: 0.5292 - val_loss: 0.1642 - val_mse: 0.0526 - val_rmse: 0.1878 Epoch 43/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4763 - mse: 0.5172 - rmse: 0.5291 - val_loss: 0.1633 - val_mse: 0.0523 - val_rmse: 0.1869 Epoch 44/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4763 - mse: 0.5172 - rmse: 0.5289 - val_loss: 0.1635 - val_mse: 0.0523 - val_rmse: 0.1866 Epoch 45/68 62/62 [==============================] - 1s 13ms/step - loss: 0.4762 - mse: 0.5170 - rmse: 0.5289 - val_loss: 0.1630 - val_mse: 0.0522 - val_rmse: 0.1859 Epoch 46/68 62/62 [==============================] - ETA: 0s - loss: 0.4759 - mse: 0.5168 - rmse: 0.52 - 1s 13ms/step - loss: 0.4759 - mse: 0.5168 - rmse: 0.5286 - val_loss: 0.1626 - val_mse: 0.0520 - val_rmse: 0.1852 Epoch 47/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4758 - mse: 0.5168 - rmse: 0.5285 - val_loss: 0.1628 - val_mse: 0.0520 - val_rmse: 0.1852 Epoch 48/68 62/62 [==============================] - 1s 15ms/step - loss: 0.4755 - mse: 0.5162 - rmse: 0.5282 - val_loss: 0.1623 - val_mse: 0.0518 - val_rmse: 0.1845 Epoch 49/68 62/62 [==============================] - 1s 15ms/step - loss: 0.4755 - mse: 0.5160 - rmse: 0.5282 - val_loss: 0.1619 - val_mse: 0.0516 - val_rmse: 0.1838 Epoch 50/68 62/62 [==============================] - 1s 13ms/step - loss: 0.4755 - mse: 0.5164 - rmse: 0.5283 - val_loss: 0.1614 - val_mse: 0.0515 - val_rmse: 0.1833 Epoch 51/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4756 - mse: 0.5163 - rmse: 0.5283 - val_loss: 0.1622 - val_mse: 0.0517 - val_rmse: 0.1836 Epoch 52/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4750 - mse: 0.5161 - rmse: 0.5278 - val_loss: 0.1618 - val_mse: 0.0514 - val_rmse: 0.1829 Epoch 53/68 62/62 [==============================] - 1s 12ms/step - loss: 0.4751 - mse: 0.5159 - rmse: 0.5279 - val_loss: 0.1612 - val_mse: 0.0514 - val_rmse: 0.1824 Epoch 54/68 62/62 [==============================] - 1s 13ms/step - loss: 0.4753 - mse: 0.5159 - rmse: 0.5279 - val_loss: 0.1621 - val_mse: 0.0516 - val_rmse: 0.1830 Epoch 55/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4747 - mse: 0.5154 - rmse: 0.5274 - val_loss: 0.1617 - val_mse: 0.0514 - val_rmse: 0.1823 Epoch 56/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4747 - mse: 0.5153 - rmse: 0.5275 - val_loss: 0.1602 - val_mse: 0.0510 - val_rmse: 0.1810 Epoch 57/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4745 - mse: 0.5149 - rmse: 0.5273 - val_loss: 0.1601 - val_mse: 0.0510 - val_rmse: 0.1808 Epoch 58/68 62/62 [==============================] - 1s 13ms/step - loss: 0.4743 - mse: 0.5147 - rmse: 0.5269 - val_loss: 0.1614 - val_mse: 0.0513 - val_rmse: 0.1817 Epoch 59/68 62/62 [==============================] - 1s 15ms/step - loss: 0.4744 - mse: 0.5142 - rmse: 0.5269 - val_loss: 0.1610 - val_mse: 0.0511 - val_rmse: 0.1810 Epoch 60/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4741 - mse: 0.5144 - rmse: 0.5267 - val_loss: 0.1595 - val_mse: 0.0508 - val_rmse: 0.1797 Epoch 61/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4740 - mse: 0.5143 - rmse: 0.5269 - val_loss: 0.1600 - val_mse: 0.0508 - val_rmse: 0.1799 Epoch 62/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4740 - mse: 0.5142 - rmse: 0.5266 - val_loss: 0.1590 - val_mse: 0.0505 - val_rmse: 0.1789 Epoch 63/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4738 - mse: 0.5139 - rmse: 0.5266 - val_loss: 0.1592 - val_mse: 0.0507 - val_rmse: 0.1791 Epoch 64/68 62/62 [==============================] - 1s 13ms/step - loss: 0.4739 - mse: 0.5141 - rmse: 0.5268 - val_loss: 0.1606 - val_mse: 0.0510 - val_rmse: 0.1799 Epoch 65/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4736 - mse: 0.5137 - rmse: 0.5264 - val_loss: 0.1614 - val_mse: 0.0512 - val_rmse: 0.1804 Epoch 66/68 62/62 [==============================] - 1s 13ms/step - loss: 0.4733 - mse: 0.5134 - rmse: 0.5262 - val_loss: 0.1598 - val_mse: 0.0507 - val_rmse: 0.1789 Epoch 67/68 62/62 [==============================] - 1s 14ms/step - loss: 0.4736 - mse: 0.5138 - rmse: 0.5265 - val_loss: 0.1606 - val_mse: 0.0510 - val_rmse: 0.1794
import matplotlib.pyplot as plt
plt.plot(history.history['loss'], label='MAE Training loss')
plt.plot(history.history['val_loss'], label='MAE Validation loss')
plt.plot(history.history['mse'], label='MSE Training loss')
plt.plot(history.history['val_mse'], label='MSE Validation loss')
plt.plot(history.history['rmse'], label='RMSE Training loss')
plt.plot(history.history['val_rmse'], label='RMSE Validation loss')
plt.legend();
X_train_pred = model.predict(X_train, verbose=0)
train_mae_loss = np.mean(np.abs(X_train_pred - X_train), axis=1)
plt.hist(train_mae_loss, bins=50)
plt.xlabel('Train MAE loss')
plt.ylabel('Number of Samples');
def evaluate_prediction(predictions, actual, model_name):
errors = predictions - actual
mse = np.square(errors).mean()
rmse = np.sqrt(mse)
mae = np.abs(errors).mean()
print(model_name + ':')
print('Mean Absolute Error: {:.4f}'.format(mae))
print('Root Mean Square Error: {:.4f}'.format(rmse))
print('Mean Square Error: {:.4f}'.format(mse))
print('')
return mae,rmse,mse
mae,rmse,mse = evaluate_prediction(X_train_pred, X_train,"LSTM")
LSTM: Mean Absolute Error: 0.2089 Root Mean Square Error: 0.4480 Mean Square Error: 0.2007
model.save(MODELFILENAME+'.h5')
#càlcul del threshold de test
def calculate_threshold(X_test, X_test_pred):
distance = np.sqrt(np.mean(np.square(X_test_pred - X_test),axis=1))
"""Sorting the scores/diffs and using a 0.80 as cutoff value to pick the threshold"""
distance.sort();
cut_off = int(0.95 * len(distance));
threshold = distance[cut_off];
return threshold
for col in columns:
print ("####################### "+col +" ###########################")
#Standardize the test data
scaler = StandardScaler()
test_cpy = test.copy()
test[col] = scaler.fit_transform(test[[col]])
#creem seqüencia amb finestra temporal per les dades de test
X_test1, y_test1 = create_sequences(test[[col]], test[col])
print(f'Testing shape: {X_test1.shape}')
#evaluem el model
eval = model.evaluate(X_test1, y_test1)
print("evaluate: ",eval)
#predim el model
X_test1_pred = model.predict(X_test1, verbose=0)
evaluate_prediction(X_test1_pred, X_test1,"LSTM")
#càlcul del mae_loss
test1_mae_loss = np.mean(np.abs(X_test1_pred - X_test1), axis=1)
test1_rmse_loss = np.sqrt(np.mean(np.square(X_test1_pred - X_test1),axis=1))
# reshaping test prediction
X_test1_predReshape = X_test1_pred.reshape((X_test1_pred.shape[0] * X_test1_pred.shape[1]), X_test1_pred.shape[2])
# reshaping test data
X_test1Reshape = X_test1.reshape((X_test1.shape[0] * X_test1.shape[1]), X_test1.shape[2])
threshold_test = calculate_threshold(X_test1Reshape,X_test1_predReshape)
test1_score_df = pd.DataFrame(test[TIME_STEPS:])
test1_score_df['loss'] = test1_rmse_loss.reshape((-1))
test1_score_df['threshold'] = threshold_test
test1_score_df['anomaly'] = test1_score_df['loss'] > test1_score_df['threshold']
test1_score_df[col] = test[TIME_STEPS:][col]
#gràfic test lost i threshold
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['loss'], name='Test loss'))
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['threshold'], name='Threshold'))
fig.update_layout(showlegend=True, title='Test loss vs. Threshold')
fig.show()
#Posem les anomalies en un array
anomalies1 = test1_score_df.loc[test1_score_df['anomaly'] == True]
anomalies1.shape
print('anomalies: ',anomalies1.shape); print();
#Gràfic dels punts i de les anomalíes amb els valors de dades transformades per verificar que la normalització que s'ha fet no distorssiona les dades
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=scaler.inverse_transform(test1_score_df[col]), name=col))
fig.add_trace(go.Scatter(x=anomalies1.index, y=scaler.inverse_transform(anomalies1[col]), mode='markers', name='Anomaly'))
fig.update_layout(showlegend=True, title='Detected anomalies')
fig.show()
print ("######################################################")
####################### PM1 ###########################
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy test[col] = scaler.fit_transform(test[[col]])
Testing shape: (744, 36, 1) 24/24 [==============================] - 0s 5ms/step - loss: 0.5508 - mse: 0.9637 - rmse: 0.6326 evaluate: [0.5507761836051941, 0.9636529088020325, 0.6325988173484802] LSTM: Mean Absolute Error: 0.1967 Root Mean Square Error: 0.5859 Mean Square Error: 0.3433
anomalies: (91, 10)
###################################################### ####################### PM25 ########################### Testing shape: (744, 36, 1)
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
24/24 [==============================] - 0s 4ms/step - loss: 0.5720 - mse: 0.9251 - rmse: 0.6538 evaluate: [0.5720363855361938, 0.9250632524490356, 0.6537909507751465] LSTM: Mean Absolute Error: 0.2081 Root Mean Square Error: 0.5401 Mean Square Error: 0.2917
anomalies: (100, 10)
###################################################### ####################### PM10 ########################### Testing shape: (744, 36, 1)
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
24/24 [==============================] - 0s 4ms/step - loss: 0.5848 - mse: 0.9099 - rmse: 0.6682 evaluate: [0.5847648978233337, 0.9098789095878601, 0.668245255947113] LSTM: Mean Absolute Error: 0.2141 Root Mean Square Error: 0.4990 Mean Square Error: 0.2490
anomalies: (58, 10)
###################################################### ####################### PM1ATM ########################### Testing shape: (744, 36, 1)
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
24/24 [==============================] - 0s 4ms/step - loss: 0.5925 - mse: 0.9338 - rmse: 0.6807 evaluate: [0.5925184488296509, 0.9337892532348633, 0.6807057857513428] LSTM: Mean Absolute Error: 0.2112 Root Mean Square Error: 0.5077 Mean Square Error: 0.2578
anomalies: (76, 10)
###################################################### ####################### PM25ATM ########################### Testing shape: (744, 36, 1) 1/24 [>.............................] - ETA: 0s - loss: 0.7542 - mse: 3.4672 - rmse: 0.94
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
24/24 [==============================] - 0s 3ms/step - loss: 0.5868 - mse: 0.9363 - rmse: 0.6745 evaluate: [0.5868412256240845, 0.9363366961479187, 0.6744985580444336] LSTM: Mean Absolute Error: 0.2086 Root Mean Square Error: 0.5193 Mean Square Error: 0.2697
anomalies: (83, 10)
###################################################### ####################### PM10ATM ########################### Testing shape: (744, 36, 1) 24/24 [==============================] - 0s 3ms/step - loss: 0.5818 - mse: 0.9079 - rmse: 0.6641
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
evaluate: [0.5817864537239075, 0.9078505039215088, 0.6640828847885132] LSTM: Mean Absolute Error: 0.2139 Root Mean Square Error: 0.5231 Mean Square Error: 0.2736
anomalies: (79, 10)
######################################################